DGMD S-14 Summer 2020 Final Project¶

# Importing the necessary libraries

​

# Built-in libraries

import os.path

import glob

import json

from datetime import datetime

import statistics

​

# Data science libraries

import pandas

import numpy

from sklearn.linear_model import LogisticRegression

from sklearn.model_selection import train_test_split

from sklearn.metrics import accuracy_score

​

# Plotting libraries

import plotly.offline as pyo

import plotly.graph_objs as go

​

# Jupyter Notebook libraries

from IPython.display import display

# Physical cube data

PHYSICAL_CUBE_PATH = os.path.join("putty_data")

​

# Conversion function

def createDataFrameFromPuttyLogFile(

    filePath: str

) -> pandas.DataFrame:

    """

    This functions takes the file path of a putty log file containing sensor tile data

    and cleans and converts the data into a pandas.DataFrame, The data in the putty log

    file must be in the following form for each line:

    A_y: int, TimeStamp: hour:minutes:seconds.microseconds

    The data will be first cleaned by this function, removing any putty log headers, 

    blank lines, and partial lines. Also, as a precaution, It will remove the final line

    even if it is valid to avoid any cases of partial lines being sent to the putty instance

    Args:

        filePath (str): the file path of the putty log fiel

    Returns:

        pandas.DataFrame: returns a dataframe in the following form:

            +-------+-------+-----------------+

            | Index |  A_y  |    TimeStamp    |

            |-------+-------+-----------------|

            |   0   | (int) | (datetime.time) |

            +-------+-------+-----------------+

    Raises:

        None

    """

    # Opening the file and reading the file data

    with open(filePath) as puttyDataFile:

        puttyData = puttyDataFile.read()

    # Splitting the file data into lines of data

    puttyData = puttyData.split("\n")

​

    ##############################

    # Filtering Process

    ##############################

​

    # Filtering out the PuTTY log header

    puttyData = list(filter(lambda d: "~=" not in d, puttyData))

​

    # Filtering out blank lines

    puttyData = list(filter(lambda d: d != "", puttyData))

​

    # Filtering out lines with no comma delimiter

    puttyData = list(filter(lambda d: "," in d, puttyData))

​

    # Filtering out partial lines (in this case ones that don't have both

    # the A_y and TimeStamp keys)

    puttyData = list(filter(lambda d: "A_y:" in d and "TimeStamp:" in d, puttyData))

​

    # Removing the final line of data, as in some cases it may be a partial line

    # but not detected by the prior filters

    puttyData = puttyData[0:len(puttyData) - 1]

​

​

    ##############################

    # Data Manipulation Process

    ##############################

​

    # In this code I am doing several operations. I am looping through every single line of

    # putty data and for each line I creating a dictionary in the form of:

    #

    # {

    #   A_y: int,

    #   TimeStamp: datetime.time

    # }

    #

    # The operations I am performing are:

    # 1) Split the line on a comma, take the left half of the line, and parsing out the A_y

    #    value and then finally converting it to an integer

    # 2) Split the line on a comma, take the right half, parse out the string that represents

    #    the time, converting that to a datetime.datetime object via .strptime() calls, and then

    #    from that datetime, converting it to a datetime.time object via the .time() calls. Since

    #    the timestamps don't include day/month/year I am only taking the time portion of it.

    puttyData = [{

        "A_y": int(line.split(", ")[0].split(":")[1].strip()),

        "TimeStamp": datetime.strptime(

            line.split(", ")[1].split(":", 1)[1].strip(),

            "%H:%M:%S.%f"

        ).time(),

    } for line in puttyData]

    # Finally, pipping the list of dictionaries into a pandas.DataFrame

    df = pandas.DataFrame(puttyData)

    return df

# Creating a list of dataframes for each of the three height throwing tests

oneFootThrowDataFrameList = [

    createDataFrameFromPuttyLogFile(filePath) 

    for filePath in 

    glob.glob(os.path.join(PHYSICAL_CUBE_PATH, "putty?.txt"))

]

​

print("Count of DataFrames for Physical 1ft Throws:", len(oneFootThrowDataFrameList))

​

twoFootThrowDataFrameList = [

    createDataFrameFromPuttyLogFile(filePath) 

    for filePath in 

    glob.glob(os.path.join(PHYSICAL_CUBE_PATH, "putty1?.txt"))

]

​

print("Count of DataFrames for Physical 2ft Throws:", len(twoFootThrowDataFrameList))

​

threeFootThrowDataFrameList = [

    createDataFrameFromPuttyLogFile(filePath) 

    for filePath in 

    glob.glob(os.path.join(PHYSICAL_CUBE_PATH, "putty2?.txt"))

]

​

print("Count of DataFrames for Physical 3ft Throws:", len(threeFootThrowDataFrameList))

​

accYAdjustment = [

    df["A_y"].median()

    for df in 

    oneFootThrowDataFrameList + twoFootThrowDataFrameList + threeFootThrowDataFrameList

]

​

accYAdjustment = statistics.mean(accYAdjustment)

# Plotting and describing function

def displayChartFromPuttyDataFrame(

    puttyDf: pandas.DataFrame, 

    testType: int, 

    testNumber: int

) -> None:

    """

    Displays a plotly chart and the descriptive statistics of a dataframe within a juypter notebook

    based on the data from putty.

    Args:

        puttyDf (pandas.DataFrame): a dataframe in the form of

            +-------+-------+-----------------+

            | Index |  A_y  |    TimeStamp    |

            |-------+-------+-----------------|

            |   0   | (int) | (datetime.time) |

            +-------+-------+-----------------+

        testType (int): represents the number of feet the cube was thrown for that test

        testNumber (int): the test number (used to determine chart title)

    """

    # Plotting the plotly chart out. This chart uses 2 lines, one of them plots the following:

    #   x=TimeStamp

    #   y=Acc_Y

    #

    # The other chart plots a moving average of the same data (currently set to 10-point MA)

    fig = go.Figure({

        # Data

        "data":[

            go.Scatter(

                x=puttyDf["TimeStamp"],

                y=puttyDf["A_y"],

                name="Acc_Y",

            ),

            go.Scatter(

                x=puttyDf["TimeStamp"],

                y=puttyDf["A_y"].rolling(window=10).mean(),

                name="10-n MA Acc_Y",

            ),

        ],

        # Chart styling features

        "layout": go.Layout(

            title="Test #{} - {} Foot Throw".format(testNumber, testType),

            titlefont={

                "size":32,

                "color":"#000000"

            },

            plot_bgcolor="#ffffff",

            xaxis={

                "gridcolor":"#dddddd",

            },

            yaxis={

                "gridcolor":"#dddddd",

                "range":[0,3000],

            }

        )

    })

    # Displaying the chart

    fig.show()

    # Displaying the dataframe inputting into this function

    display(df.describe())

​

# Plotting out all charts from the three different test cases using the same

# scale for each of the different test cases

i = 0

for df in oneFootThrowDataFrameList:

    i += 1

    displayChartFromPuttyDataFrame(df, 1, i)

for df in twoFootThrowDataFrameList:

    i += 1

    displayChartFromPuttyDataFrame(df, 2, i)

for df in threeFootThrowDataFrameList:

    i += 1

    displayChartFromPuttyDataFrame(df, 3, i)